{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/client/session.py:1751: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.compat.v1 import ConfigProto\n",
    "from tensorflow.compat.v1 import InteractiveSession\n",
    "\n",
    "config = ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.8\n",
    "config.gpu_options.allow_growth = True\n",
    "session = InteractiveSession(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.utils import class_weight\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_pd(train_path,test_path):\n",
    "    train=pd.read_csv(train_path)\n",
    "    test=pd.read_csv(test_path)\n",
    "    train.columns=[x for x in range(188)]\n",
    "    test.columns=[x for x in range(188)]\n",
    "    return pd.concat([train,test], axis=0, join='inner').sort_index()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "mit_train_path=\"../input/heartbeat/mitbih_train.csv\"\n",
    "mit_test_path=\"../input/heartbeat/mitbih_test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>178</th>\n",
       "      <th>179</th>\n",
       "      <th>180</th>\n",
       "      <th>181</th>\n",
       "      <th>182</th>\n",
       "      <th>183</th>\n",
       "      <th>184</th>\n",
       "      <th>185</th>\n",
       "      <th>186</th>\n",
       "      <th>187</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.960114</td>\n",
       "      <td>0.863248</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.196581</td>\n",
       "      <td>0.094017</td>\n",
       "      <td>0.125356</td>\n",
       "      <td>0.099715</td>\n",
       "      <td>0.088319</td>\n",
       "      <td>0.074074</td>\n",
       "      <td>0.082621</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.908425</td>\n",
       "      <td>0.783883</td>\n",
       "      <td>0.531136</td>\n",
       "      <td>0.362637</td>\n",
       "      <td>0.366300</td>\n",
       "      <td>0.344322</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.296703</td>\n",
       "      <td>0.300366</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.730088</td>\n",
       "      <td>0.212389</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.119469</td>\n",
       "      <td>0.101770</td>\n",
       "      <td>0.101770</td>\n",
       "      <td>0.110619</td>\n",
       "      <td>0.123894</td>\n",
       "      <td>0.115044</td>\n",
       "      <td>0.132743</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.659459</td>\n",
       "      <td>0.186486</td>\n",
       "      <td>0.070270</td>\n",
       "      <td>0.070270</td>\n",
       "      <td>0.059459</td>\n",
       "      <td>0.056757</td>\n",
       "      <td>0.043243</td>\n",
       "      <td>0.054054</td>\n",
       "      <td>0.045946</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.925414</td>\n",
       "      <td>0.665746</td>\n",
       "      <td>0.541436</td>\n",
       "      <td>0.276243</td>\n",
       "      <td>0.196133</td>\n",
       "      <td>0.077348</td>\n",
       "      <td>0.071823</td>\n",
       "      <td>0.060773</td>\n",
       "      <td>0.066298</td>\n",
       "      <td>0.058011</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 188 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1         2         3         4         5         6    \\\n",
       "0  0.960114  0.863248  0.461538  0.196581  0.094017  0.125356  0.099715   \n",
       "0  0.908425  0.783883  0.531136  0.362637  0.366300  0.344322  0.333333   \n",
       "1  0.730088  0.212389  0.000000  0.119469  0.101770  0.101770  0.110619   \n",
       "1  1.000000  0.659459  0.186486  0.070270  0.070270  0.059459  0.056757   \n",
       "2  0.925414  0.665746  0.541436  0.276243  0.196133  0.077348  0.071823   \n",
       "\n",
       "        7         8         9    ...  178  179  180  181  182  183  184  185  \\\n",
       "0  0.088319  0.074074  0.082621  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "0  0.307692  0.296703  0.300366  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1  0.123894  0.115044  0.132743  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1  0.043243  0.054054  0.045946  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2  0.060773  0.066298  0.058011  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "   186  187  \n",
       "0  0.0  0.0  \n",
       "0  0.0  0.0  \n",
       "1  0.0  0.0  \n",
       "1  0.0  0.0  \n",
       "2  0.0  0.0  \n",
       "\n",
       "[5 rows x 188 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mit= create_pd(mit_train_path,mit_test_path)\n",
    "mit.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_k_folds_column(df):\n",
    "    df.loc[:,'kfold']=-1\n",
    "    df=df.sample(frac=1).reset_index(drop=True)\n",
    "    y=df.loc[:,187].values\n",
    "    kf=StratifiedKFold(n_splits=5)\n",
    "    for fold,(target,index) in enumerate(kf.split(X=df,y=y)):\n",
    "        df.loc[index,'kfold']=fold\n",
    "    return df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "mit= create_k_folds_column(mit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The MITBIH dataset is constituted of 109446 beats, labeled with 5 different classes :\n",
    "\n",
    "'N': 0, 'S': 1, 'V': 2, 'F': 3, 'Q': 4\n",
    "\n",
    "N : Non-ecotic beats (normal beat) \n",
    "\n",
    "S : Supraventricular ectopic beats \n",
    "\n",
    "V : Ventricular ectopic beats\n",
    "\n",
    "F : Fusion Beats \n",
    "\n",
    "Q : Unknown Beats\n",
    "\n",
    "The PTBHB dataset is constituted of 14552 beats, labeled with two different classes :\n",
    "\n",
    "'0' for normal beat\n",
    "'1' for abnormal beat (Myocardial infarction)\n",
    "\n",
    "All the beats are recorded with 187 points. The shorter beats are padded with zeros to reach 187."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>179</th>\n",
       "      <th>180</th>\n",
       "      <th>181</th>\n",
       "      <th>182</th>\n",
       "      <th>183</th>\n",
       "      <th>184</th>\n",
       "      <th>185</th>\n",
       "      <th>186</th>\n",
       "      <th>187</th>\n",
       "      <th>kfold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.659794</td>\n",
       "      <td>0.068729</td>\n",
       "      <td>0.099656</td>\n",
       "      <td>0.195876</td>\n",
       "      <td>0.182131</td>\n",
       "      <td>0.147766</td>\n",
       "      <td>0.137457</td>\n",
       "      <td>0.140893</td>\n",
       "      <td>0.147766</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.941909</td>\n",
       "      <td>0.468880</td>\n",
       "      <td>0.338174</td>\n",
       "      <td>0.327801</td>\n",
       "      <td>0.325726</td>\n",
       "      <td>0.259336</td>\n",
       "      <td>0.230290</td>\n",
       "      <td>0.178423</td>\n",
       "      <td>0.139004</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.739535</td>\n",
       "      <td>0.316279</td>\n",
       "      <td>0.058915</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.103876</td>\n",
       "      <td>0.139535</td>\n",
       "      <td>0.103876</td>\n",
       "      <td>0.105426</td>\n",
       "      <td>0.106977</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.077519</td>\n",
       "      <td>0.067183</td>\n",
       "      <td>0.095607</td>\n",
       "      <td>0.162791</td>\n",
       "      <td>0.258398</td>\n",
       "      <td>0.317829</td>\n",
       "      <td>0.330749</td>\n",
       "      <td>0.361757</td>\n",
       "      <td>0.418605</td>\n",
       "      <td>0.490956</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.955556</td>\n",
       "      <td>0.490370</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.189630</td>\n",
       "      <td>0.241481</td>\n",
       "      <td>0.213333</td>\n",
       "      <td>0.217778</td>\n",
       "      <td>0.232593</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 189 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  1.000000  0.659794  0.068729  0.099656  0.195876  0.182131  0.147766   \n",
       "1  1.000000  0.941909  0.468880  0.338174  0.327801  0.325726  0.259336   \n",
       "2  1.000000  0.739535  0.316279  0.058915  0.000000  0.103876  0.139535   \n",
       "3  0.077519  0.067183  0.095607  0.162791  0.258398  0.317829  0.330749   \n",
       "4  1.000000  0.955556  0.490370  0.000000  0.013333  0.189630  0.241481   \n",
       "\n",
       "          7         8         9  ...  179  180  181  182  183  184  185  186  \\\n",
       "0  0.137457  0.140893  0.147766  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1  0.230290  0.178423  0.139004  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2  0.103876  0.105426  0.106977  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3  0.361757  0.418605  0.490956  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "4  0.213333  0.217778  0.232593  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "   187  kfold  \n",
       "0  0.0      0  \n",
       "1  0.0      0  \n",
       "2  0.0      0  \n",
       "3  2.0      0  \n",
       "4  0.0      0  \n",
       "\n",
       "[5 rows x 189 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mit.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    90587\n",
       "4     8039\n",
       "2     7236\n",
       "1     2779\n",
       "3      803\n",
       "Name: 187, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mit.loc[:,187].astype('int').value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, Activation, Flatten, Convolution1D, Dropout,MaxPooling1D,GlobalAveragePooling1D\n",
    "from tensorflow.keras import Model, layers,Sequential,regularizers\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import EarlyStopping,ReduceLROnPlateau\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import LearningRateScheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_model(X_train):\n",
    "    model= Sequential()\n",
    "    model.add(Convolution1D(32,5,activation='relu',input_shape=(187,1)))\n",
    "    model.add(Convolution1D(64,5,activation='relu'))         \n",
    "    model.add(MaxPooling1D(3))\n",
    "    model.add(Convolution1D(128, 3, activation='relu'))\n",
    "    model.add(Convolution1D(256, 3, activation='relu'))\n",
    "    model.add(GlobalAveragePooling1D())\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(1024,activation='relu'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(256,activation='relu'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(32,activation='relu'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(5,activation='softmax'))\n",
    "    model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy',tf.keras.metrics.AUC(name='auc')])\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_data(train,valid):\n",
    "    X_train=np.asarray(train.iloc[:,:187].values)\n",
    "    y_train=train.iloc[:,187].values\n",
    "    X_valid=np.asarray(valid.iloc[:,:187].values)\n",
    "    y_valid=valid.iloc[:,187].values\n",
    "    X_train=tf.expand_dims(X_train, axis=2)\n",
    "    X_valid=tf.expand_dims(X_valid, axis=2)\n",
    "    y_train=to_categorical(y_train)\n",
    "    y_valid=to_categorical(y_valid)\n",
    "    return X_train,y_train,X_valid,y_valid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Fitting the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "Epochs=100\n",
    "Batch_size=64\n",
    "my_callbacks = [EarlyStopping(patience=3,monitor='val_loss', mode='min',restore_best_weights=True),\n",
    "               ReduceLROnPlateau(monitor='val_loss', factor=0.1,patience=2, min_lr=0.00001, mode='auto')]\n",
    "dict_acc={}\n",
    "dict_acc2={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_train(fold):\n",
    "    train=mit[mit[\"kfold\"]!=fold].reset_index(drop=True)\n",
    "    valid=mit[mit[\"kfold\"]==fold].reset_index(drop=True)\n",
    "    X_train,y_train,X_valid,y_valid=training_data(train,valid)\n",
    "    model=make_model(X_train)\n",
    "    history = model.fit(X_train,y_train,validation_split=0.1,batch_size=Batch_size,epochs=Epochs,callbacks=my_callbacks)\n",
    "    model.save(f'model{fold}.h5')\n",
    "    results = model.evaluate(X_valid, y_valid)\n",
    "    print(\"Test Accuracy: {:.2f}%\".format(results[1] * 100))\n",
    "    print(\"     Test AUC: {:.4f}\".format(results[2]))\n",
    "    dict_acc[f\"{i}\"]= \"Test Accuracy: {:.2f}%\".format(results[1] * 100) \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0-fold trained\n",
      "Epoch 1/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.4242 - accuracy: 0.8799 - auc: 0.9734 - val_loss: 0.2319 - val_accuracy: 0.9379 - val_auc: 0.9905\n",
      "Epoch 2/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.2244 - accuracy: 0.9420 - auc: 0.9898 - val_loss: 0.1728 - val_accuracy: 0.9542 - val_auc: 0.9947\n",
      "Epoch 3/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1813 - accuracy: 0.9527 - auc: 0.9924 - val_loss: 0.1479 - val_accuracy: 0.9563 - val_auc: 0.9955\n",
      "Epoch 4/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1600 - accuracy: 0.9584 - auc: 0.9939 - val_loss: 0.1321 - val_accuracy: 0.9611 - val_auc: 0.9961\n",
      "Epoch 5/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1461 - accuracy: 0.9616 - auc: 0.9945 - val_loss: 0.1242 - val_accuracy: 0.9647 - val_auc: 0.9963\n",
      "Epoch 6/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1324 - accuracy: 0.9651 - auc: 0.9952 - val_loss: 0.1249 - val_accuracy: 0.9680 - val_auc: 0.9953\n",
      "Epoch 7/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1215 - accuracy: 0.9681 - auc: 0.9956 - val_loss: 0.1009 - val_accuracy: 0.9720 - val_auc: 0.9974\n",
      "Epoch 8/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1138 - accuracy: 0.9703 - auc: 0.9962 - val_loss: 0.0999 - val_accuracy: 0.9727 - val_auc: 0.9974\n",
      "Epoch 9/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1052 - accuracy: 0.9726 - auc: 0.9966 - val_loss: 0.0977 - val_accuracy: 0.9719 - val_auc: 0.9975\n",
      "Epoch 10/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.1009 - accuracy: 0.9734 - auc: 0.9969 - val_loss: 0.0922 - val_accuracy: 0.9727 - val_auc: 0.9979\n",
      "Epoch 11/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0981 - accuracy: 0.9743 - auc: 0.9970 - val_loss: 0.1058 - val_accuracy: 0.9734 - val_auc: 0.9958\n",
      "Epoch 12/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0916 - accuracy: 0.9759 - auc: 0.9972 - val_loss: 0.0891 - val_accuracy: 0.9757 - val_auc: 0.9978\n",
      "Epoch 13/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0896 - accuracy: 0.9759 - auc: 0.9973 - val_loss: 0.0819 - val_accuracy: 0.9773 - val_auc: 0.9979\n",
      "Epoch 14/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0850 - accuracy: 0.9775 - auc: 0.9975 - val_loss: 0.0872 - val_accuracy: 0.9758 - val_auc: 0.9976\n",
      "Epoch 15/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0847 - accuracy: 0.9775 - auc: 0.9976 - val_loss: 0.0912 - val_accuracy: 0.9740 - val_auc: 0.9974\n",
      "Epoch 16/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0671 - accuracy: 0.9820 - auc: 0.9983 - val_loss: 0.0728 - val_accuracy: 0.9797 - val_auc: 0.9985\n",
      "Epoch 17/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0626 - accuracy: 0.9830 - auc: 0.9985 - val_loss: 0.0727 - val_accuracy: 0.9805 - val_auc: 0.9984\n",
      "Epoch 18/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0627 - accuracy: 0.9834 - auc: 0.9985 - val_loss: 0.0752 - val_accuracy: 0.9794 - val_auc: 0.9981\n",
      "Epoch 19/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0601 - accuracy: 0.9839 - auc: 0.9986 - val_loss: 0.0725 - val_accuracy: 0.9808 - val_auc: 0.9982\n",
      "Epoch 20/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0588 - accuracy: 0.9841 - auc: 0.9987 - val_loss: 0.0723 - val_accuracy: 0.9805 - val_auc: 0.9982\n",
      "Epoch 21/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0577 - accuracy: 0.9843 - auc: 0.9987 - val_loss: 0.0729 - val_accuracy: 0.9805 - val_auc: 0.9982\n",
      "Epoch 22/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0582 - accuracy: 0.9843 - auc: 0.9986 - val_loss: 0.0730 - val_accuracy: 0.9805 - val_auc: 0.9981\n",
      "Epoch 23/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0575 - accuracy: 0.9845 - auc: 0.9988 - val_loss: 0.0739 - val_accuracy: 0.9798 - val_auc: 0.9981\n",
      "685/685 [==============================] - 2s 3ms/step - loss: 0.0620 - accuracy: 0.9828 - auc: 0.9985\n",
      "Test Accuracy: 98.28%\n",
      "     Test AUC: 0.9985\n",
      "_______________________________\n",
      "_______________________________\n",
      "1-fold trained\n",
      "Epoch 1/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.4294 - accuracy: 0.8794 - auc: 0.9730 - val_loss: 0.2588 - val_accuracy: 0.9348 - val_auc: 0.9913\n",
      "Epoch 2/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.2215 - accuracy: 0.9419 - auc: 0.9901 - val_loss: 0.1613 - val_accuracy: 0.9556 - val_auc: 0.9947\n",
      "Epoch 3/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1772 - accuracy: 0.9549 - auc: 0.9927 - val_loss: 0.1445 - val_accuracy: 0.9601 - val_auc: 0.9943\n",
      "Epoch 4/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.1570 - accuracy: 0.9592 - auc: 0.9940 - val_loss: 0.1567 - val_accuracy: 0.9577 - val_auc: 0.9933\n",
      "Epoch 5/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1444 - accuracy: 0.9611 - auc: 0.9949 - val_loss: 0.1307 - val_accuracy: 0.9652 - val_auc: 0.9960\n",
      "Epoch 6/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1301 - accuracy: 0.9650 - auc: 0.9955 - val_loss: 0.1191 - val_accuracy: 0.9681 - val_auc: 0.9959\n",
      "Epoch 7/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1223 - accuracy: 0.9679 - auc: 0.9957 - val_loss: 0.1055 - val_accuracy: 0.9713 - val_auc: 0.9968\n",
      "Epoch 8/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1089 - accuracy: 0.9713 - auc: 0.9964 - val_loss: 0.0987 - val_accuracy: 0.9730 - val_auc: 0.9972\n",
      "Epoch 9/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1045 - accuracy: 0.9719 - auc: 0.9966 - val_loss: 0.0870 - val_accuracy: 0.9750 - val_auc: 0.9977\n",
      "Epoch 10/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0991 - accuracy: 0.9732 - auc: 0.9969 - val_loss: 0.0948 - val_accuracy: 0.9738 - val_auc: 0.9978\n",
      "Epoch 11/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0928 - accuracy: 0.9752 - auc: 0.9973 - val_loss: 0.0934 - val_accuracy: 0.9729 - val_auc: 0.9973\n",
      "Epoch 12/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0761 - accuracy: 0.9797 - auc: 0.9980 - val_loss: 0.0750 - val_accuracy: 0.9781 - val_auc: 0.9983\n",
      "Epoch 13/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0714 - accuracy: 0.9806 - auc: 0.9982 - val_loss: 0.0743 - val_accuracy: 0.9788 - val_auc: 0.9983\n",
      "Epoch 14/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0693 - accuracy: 0.9811 - auc: 0.9983 - val_loss: 0.0717 - val_accuracy: 0.9794 - val_auc: 0.9985\n",
      "Epoch 15/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0688 - accuracy: 0.9812 - auc: 0.9983 - val_loss: 0.0744 - val_accuracy: 0.9790 - val_auc: 0.9981\n",
      "Epoch 16/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0666 - accuracy: 0.9820 - auc: 0.9984 - val_loss: 0.0718 - val_accuracy: 0.9798 - val_auc: 0.9983\n",
      "Epoch 17/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0641 - accuracy: 0.9827 - auc: 0.9984 - val_loss: 0.0705 - val_accuracy: 0.9802 - val_auc: 0.9984\n",
      "Epoch 18/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0643 - accuracy: 0.9828 - auc: 0.9984 - val_loss: 0.0707 - val_accuracy: 0.9801 - val_auc: 0.9984\n",
      "Epoch 19/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0641 - accuracy: 0.9827 - auc: 0.9984 - val_loss: 0.0701 - val_accuracy: 0.9806 - val_auc: 0.9984\n",
      "Epoch 20/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0637 - accuracy: 0.9828 - auc: 0.9984 - val_loss: 0.0703 - val_accuracy: 0.9804 - val_auc: 0.9984\n",
      "Epoch 21/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0634 - accuracy: 0.9832 - auc: 0.9984 - val_loss: 0.0704 - val_accuracy: 0.9806 - val_auc: 0.9983\n",
      "Epoch 22/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0635 - accuracy: 0.9827 - auc: 0.9985 - val_loss: 0.0699 - val_accuracy: 0.9807 - val_auc: 0.9984\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0635 - accuracy: 0.9826 - auc: 0.9985 - val_loss: 0.0703 - val_accuracy: 0.9804 - val_auc: 0.9983\n",
      "Epoch 24/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0619 - accuracy: 0.9832 - auc: 0.9985 - val_loss: 0.0702 - val_accuracy: 0.9805 - val_auc: 0.9984\n",
      "Epoch 25/100\n",
      "1232/1232 [==============================] - 9s 7ms/step - loss: 0.0622 - accuracy: 0.9831 - auc: 0.9985 - val_loss: 0.0702 - val_accuracy: 0.9807 - val_auc: 0.9984\n",
      "685/685 [==============================] - 2s 3ms/step - loss: 0.0707 - accuracy: 0.9816 - auc: 0.9977\n",
      "Test Accuracy: 98.16%\n",
      "     Test AUC: 0.9977\n",
      "_______________________________\n",
      "_______________________________\n",
      "2-fold trained\n",
      "Epoch 1/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.4055 - accuracy: 0.8856 - auc: 0.9755 - val_loss: 0.2497 - val_accuracy: 0.9326 - val_auc: 0.9890\n",
      "Epoch 2/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.2167 - accuracy: 0.9430 - auc: 0.9903 - val_loss: 0.1640 - val_accuracy: 0.9521 - val_auc: 0.9948\n",
      "Epoch 3/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1753 - accuracy: 0.9544 - auc: 0.9928 - val_loss: 0.1473 - val_accuracy: 0.9574 - val_auc: 0.9953\n",
      "Epoch 4/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1602 - accuracy: 0.9572 - auc: 0.9938 - val_loss: 0.1400 - val_accuracy: 0.9595 - val_auc: 0.9956\n",
      "Epoch 5/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1442 - accuracy: 0.9613 - auc: 0.9949 - val_loss: 0.1176 - val_accuracy: 0.9672 - val_auc: 0.9972\n",
      "Epoch 6/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1318 - accuracy: 0.9650 - auc: 0.9952 - val_loss: 0.1292 - val_accuracy: 0.9648 - val_auc: 0.9951\n",
      "Epoch 7/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1187 - accuracy: 0.9689 - auc: 0.9959 - val_loss: 0.1005 - val_accuracy: 0.9732 - val_auc: 0.9975\n",
      "Epoch 8/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.1112 - accuracy: 0.9711 - auc: 0.9962 - val_loss: 0.1049 - val_accuracy: 0.9718 - val_auc: 0.9968\n",
      "Epoch 9/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1048 - accuracy: 0.9720 - auc: 0.9967 - val_loss: 0.1090 - val_accuracy: 0.9725 - val_auc: 0.9955\n",
      "Epoch 10/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0844 - accuracy: 0.9773 - auc: 0.9976 - val_loss: 0.0841 - val_accuracy: 0.9761 - val_auc: 0.9980\n",
      "Epoch 11/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0801 - accuracy: 0.9788 - auc: 0.9978 - val_loss: 0.0824 - val_accuracy: 0.9774 - val_auc: 0.9979\n",
      "Epoch 12/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0783 - accuracy: 0.9795 - auc: 0.9979 - val_loss: 0.0796 - val_accuracy: 0.9770 - val_auc: 0.9980\n",
      "Epoch 13/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0776 - accuracy: 0.9796 - auc: 0.9978 - val_loss: 0.0771 - val_accuracy: 0.9767 - val_auc: 0.9983\n",
      "Epoch 14/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0743 - accuracy: 0.9803 - auc: 0.9980 - val_loss: 0.0787 - val_accuracy: 0.9778 - val_auc: 0.9980\n",
      "Epoch 15/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0737 - accuracy: 0.9801 - auc: 0.9981 - val_loss: 0.0812 - val_accuracy: 0.9774 - val_auc: 0.9979\n",
      "Epoch 16/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.0712 - accuracy: 0.9806 - auc: 0.9982 - val_loss: 0.0779 - val_accuracy: 0.9777 - val_auc: 0.9981\n",
      "685/685 [==============================] - 2s 3ms/step - loss: 0.0816 - accuracy: 0.9779 - auc: 0.9977\n",
      "Test Accuracy: 97.79%\n",
      "     Test AUC: 0.9977\n",
      "_______________________________\n",
      "_______________________________\n",
      "3-fold trained\n",
      "Epoch 1/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.4300 - accuracy: 0.8769 - auc: 0.9725 - val_loss: 0.2354 - val_accuracy: 0.9332 - val_auc: 0.9904\n",
      "Epoch 2/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.2231 - accuracy: 0.9407 - auc: 0.9901 - val_loss: 0.1638 - val_accuracy: 0.9536 - val_auc: 0.9951\n",
      "Epoch 3/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1786 - accuracy: 0.9539 - auc: 0.9927 - val_loss: 0.1478 - val_accuracy: 0.9577 - val_auc: 0.9951\n",
      "Epoch 4/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1552 - accuracy: 0.9586 - auc: 0.9941 - val_loss: 0.1338 - val_accuracy: 0.9627 - val_auc: 0.9947\n",
      "Epoch 5/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1427 - accuracy: 0.9619 - auc: 0.9947 - val_loss: 0.1253 - val_accuracy: 0.9665 - val_auc: 0.9969\n",
      "Epoch 6/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1254 - accuracy: 0.9670 - auc: 0.9955 - val_loss: 0.1198 - val_accuracy: 0.9662 - val_auc: 0.9956\n",
      "Epoch 7/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.1154 - accuracy: 0.9700 - auc: 0.9959 - val_loss: 0.1011 - val_accuracy: 0.9717 - val_auc: 0.9972\n",
      "Epoch 8/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1074 - accuracy: 0.9718 - auc: 0.9964 - val_loss: 0.1168 - val_accuracy: 0.9684 - val_auc: 0.9974\n",
      "Epoch 9/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1033 - accuracy: 0.9728 - auc: 0.9966 - val_loss: 0.0974 - val_accuracy: 0.9719 - val_auc: 0.9975\n",
      "Epoch 10/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0957 - accuracy: 0.9745 - auc: 0.9970 - val_loss: 0.0863 - val_accuracy: 0.9758 - val_auc: 0.9979\n",
      "Epoch 11/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0929 - accuracy: 0.9750 - auc: 0.9972 - val_loss: 0.0949 - val_accuracy: 0.9735 - val_auc: 0.9975\n",
      "Epoch 12/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0888 - accuracy: 0.9765 - auc: 0.9972 - val_loss: 0.0793 - val_accuracy: 0.9778 - val_auc: 0.9979\n",
      "Epoch 13/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0836 - accuracy: 0.9777 - auc: 0.9975 - val_loss: 0.1008 - val_accuracy: 0.9765 - val_auc: 0.9962\n",
      "Epoch 14/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0805 - accuracy: 0.9787 - auc: 0.9977 - val_loss: 0.0807 - val_accuracy: 0.9777 - val_auc: 0.9976\n",
      "Epoch 15/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0650 - accuracy: 0.9823 - auc: 0.9983 - val_loss: 0.0690 - val_accuracy: 0.9805 - val_auc: 0.9982\n",
      "Epoch 16/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0611 - accuracy: 0.9835 - auc: 0.9986 - val_loss: 0.0692 - val_accuracy: 0.9809 - val_auc: 0.9981\n",
      "Epoch 17/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0581 - accuracy: 0.9839 - auc: 0.9986 - val_loss: 0.0687 - val_accuracy: 0.9802 - val_auc: 0.9981\n",
      "Epoch 18/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0559 - accuracy: 0.9845 - auc: 0.9987 - val_loss: 0.0694 - val_accuracy: 0.9807 - val_auc: 0.9979\n",
      "Epoch 19/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0553 - accuracy: 0.9848 - auc: 0.9988 - val_loss: 0.0662 - val_accuracy: 0.9815 - val_auc: 0.9984\n",
      "Epoch 20/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0555 - accuracy: 0.9848 - auc: 0.9987 - val_loss: 0.0676 - val_accuracy: 0.9813 - val_auc: 0.9982\n",
      "Epoch 21/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0534 - accuracy: 0.9852 - auc: 0.9988 - val_loss: 0.0647 - val_accuracy: 0.9821 - val_auc: 0.9983\n",
      "Epoch 22/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0520 - accuracy: 0.9853 - auc: 0.9988 - val_loss: 0.0652 - val_accuracy: 0.9810 - val_auc: 0.9983\n",
      "Epoch 23/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0521 - accuracy: 0.9858 - auc: 0.9988 - val_loss: 0.0650 - val_accuracy: 0.9813 - val_auc: 0.9982\n",
      "Epoch 24/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0501 - accuracy: 0.9859 - auc: 0.9989 - val_loss: 0.0652 - val_accuracy: 0.9816 - val_auc: 0.9983\n",
      "685/685 [==============================] - 3s 4ms/step - loss: 0.0604 - accuracy: 0.9828 - auc: 0.9984\n",
      "Test Accuracy: 98.28%\n",
      "     Test AUC: 0.9984\n",
      "_______________________________\n",
      "_______________________________\n",
      "4-fold trained\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.4297 - accuracy: 0.8745 - auc: 0.9730 - val_loss: 0.2385 - val_accuracy: 0.9343 - val_auc: 0.9894\n",
      "Epoch 2/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.2230 - accuracy: 0.9407 - auc: 0.9902 - val_loss: 0.1682 - val_accuracy: 0.9551 - val_auc: 0.9939\n",
      "Epoch 3/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1784 - accuracy: 0.9523 - auc: 0.9929 - val_loss: 0.1454 - val_accuracy: 0.9611 - val_auc: 0.9950\n",
      "Epoch 4/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1555 - accuracy: 0.9581 - auc: 0.9941 - val_loss: 0.1349 - val_accuracy: 0.9647 - val_auc: 0.9947\n",
      "Epoch 5/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1391 - accuracy: 0.9626 - auc: 0.9949 - val_loss: 0.1181 - val_accuracy: 0.9665 - val_auc: 0.9964\n",
      "Epoch 6/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1261 - accuracy: 0.9661 - auc: 0.9956 - val_loss: 0.1095 - val_accuracy: 0.9710 - val_auc: 0.9966\n",
      "Epoch 7/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1178 - accuracy: 0.9684 - auc: 0.9961 - val_loss: 0.1086 - val_accuracy: 0.9714 - val_auc: 0.9960\n",
      "Epoch 8/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.1105 - accuracy: 0.9706 - auc: 0.9962 - val_loss: 0.0966 - val_accuracy: 0.9745 - val_auc: 0.9973\n",
      "Epoch 9/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.1046 - accuracy: 0.9719 - auc: 0.9966 - val_loss: 0.0960 - val_accuracy: 0.9724 - val_auc: 0.9976\n",
      "Epoch 10/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0998 - accuracy: 0.9739 - auc: 0.9968 - val_loss: 0.0927 - val_accuracy: 0.9745 - val_auc: 0.9966\n",
      "Epoch 11/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0955 - accuracy: 0.9745 - auc: 0.9970 - val_loss: 0.0902 - val_accuracy: 0.9744 - val_auc: 0.9976\n",
      "Epoch 12/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0911 - accuracy: 0.9759 - auc: 0.9972 - val_loss: 0.0908 - val_accuracy: 0.9754 - val_auc: 0.9973\n",
      "Epoch 13/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.0894 - accuracy: 0.9758 - auc: 0.9974 - val_loss: 0.0817 - val_accuracy: 0.9783 - val_auc: 0.9977\n",
      "Epoch 14/100\n",
      "1232/1232 [==============================] - 8s 7ms/step - loss: 0.0835 - accuracy: 0.9773 - auc: 0.9976 - val_loss: 0.0836 - val_accuracy: 0.9770 - val_auc: 0.9984\n",
      "Epoch 15/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0812 - accuracy: 0.9783 - auc: 0.9977 - val_loss: 0.0910 - val_accuracy: 0.9749 - val_auc: 0.9978\n",
      "Epoch 16/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0656 - accuracy: 0.9822 - auc: 0.9985 - val_loss: 0.0732 - val_accuracy: 0.9813 - val_auc: 0.9979\n",
      "Epoch 17/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0624 - accuracy: 0.9832 - auc: 0.9984 - val_loss: 0.0711 - val_accuracy: 0.9809 - val_auc: 0.9981\n",
      "Epoch 18/100\n",
      "1232/1232 [==============================] - 7s 6ms/step - loss: 0.0596 - accuracy: 0.9838 - auc: 0.9985 - val_loss: 0.0728 - val_accuracy: 0.9813 - val_auc: 0.9979\n",
      "Epoch 19/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0583 - accuracy: 0.9838 - auc: 0.9986 - val_loss: 0.0736 - val_accuracy: 0.9810 - val_auc: 0.9978\n",
      "Epoch 20/100\n",
      "1232/1232 [==============================] - 8s 6ms/step - loss: 0.0564 - accuracy: 0.9846 - auc: 0.9987 - val_loss: 0.0712 - val_accuracy: 0.9815 - val_auc: 0.9981\n",
      "684/684 [==============================] - 2s 3ms/step - loss: 0.0694 - accuracy: 0.9815 - auc: 0.9981\n",
      "Test Accuracy: 98.15%\n",
      "     Test AUC: 0.9981\n",
      "_______________________________\n",
      "_______________________________\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    print(f\"{i}-fold trained\",sep=\"/n\")\n",
    "    run_train(i)\n",
    "    print(\"_______________________________\",sep='/n')\n",
    "    print(\"_______________________________\",sep='/n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'0': 'Test Accuracy: 98.28%', '1': 'Test Accuracy: 98.16%', '2': 'Test Accuracy: 97.79%', '3': 'Test Accuracy: 98.28%', '4': 'Test Accuracy: 98.15%'}\n"
     ]
    }
   ],
   "source": [
    "print(dict_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model trained with Kfold 3 is with best performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "mit_model=tf.keras.models.load_model('model3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_24 (Conv1D)           (None, 183, 32)           192       \n",
      "_________________________________________________________________\n",
      "conv1d_25 (Conv1D)           (None, 179, 64)           10304     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_6 (MaxPooling1 (None, 59, 64)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_26 (Conv1D)           (None, 57, 128)           24704     \n",
      "_________________________________________________________________\n",
      "conv1d_27 (Conv1D)           (None, 55, 256)           98560     \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_6 ( (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dropout_24 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "flatten_6 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 1024)              263168    \n",
      "_________________________________________________________________\n",
      "dropout_25 (Dropout)         (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_25 (Dense)             (None, 256)               262400    \n",
      "_________________________________________________________________\n",
      "dropout_26 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_26 (Dense)             (None, 32)                8224      \n",
      "_________________________________________________________________\n",
      "dropout_27 (Dropout)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 5)                 165       \n",
      "=================================================================\n",
      "Total params: 667,717\n",
      "Trainable params: 667,717\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "mit_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=mit_model.predict(X_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalized confusion matrix\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqkAAALICAYAAABcjmk4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd5xU5fXH8e9ZlkWQqghsQaVJWaQJSCyxoAICYmLBXhNLNIglxpSfJjFRoyb2rlETFYkVQWAxxq50sBdWQdlCFwRBF4bz+2OGdXbZMpTZeTbzeec1L3fmPvfeM3OZzdlz7nOvubsAAACAkGSkOgAAAACgMpJUAAAABIckFQAAAMEhSQUAAEBwSFIBAAAQnMxUBwAAAICtNWi+l/umDakOQ5LkG5YXuPvQutwnSSoAAECAfNMGNep6YqrDkCR9N/+u1nW9T9r9AAAACA5JKgAAAIJDux8AACBIJln61hPT950DAAAgWFRSAQAAQmSSzFIdRcpQSQUAAEBwSFIBAAAQHNr9AAAAoWLiFAAAABAOklQAAAAEh3Y/AABAqJjdDwAAAISDSioAAECQuOMUAAAAEBSSVAAAAASHdj8AAEComDgFAAAAhIMkFQAAAMGh3Q8AABAiE7P7AQAAgJCQpAIAACA4tPsBAACCZMzuBwAAAEJCJRUAACBUTJwCAAAAwkGSCgAAgODQ7gcAAAgVE6cAAACAcJCkAgAAIDi0+wEAAIJkzO4HAAAAQkIlFQAAIEQmJk4BAAAAISFJBQAAQHBo9wMAAISKiVMAAABAOEhSAQAAEBza/QAAAEHiOqkAAABAUKikAgAAhCqD66QCAAAAwSBJBQAAQHBo9wMAAITIxMQpAAAAICQkqQAAAAgO7X4AAIBQGbP7AQAAgGCQpAIAACA4tPsBAACCxG1RAQAAgKBQSQUAAAgVE6cAAACAcJCkAgAAIDi0+wEAAELFxCkAAAAgHCSpAAAACA5JKpACZtbYzCaa2Roze2oHtnOqmU3bmbGlipkdbGafhrI/M9vbzNzM6v1pUWb2qpn9LPZzUv7NmNlvzezBnb1dIK2ZhfNIAZJUoAZmdoqZzTazdWZWamZTzOygnbDp4yW1lbS7u5+wvRtx98fd/aidEE9SxZK9zjWNcfc33L1rXcVUeX9mtsjMjqir/afKzvg3Y2aHmllRpe1e5+4/27HoAOAH9b5CACSLmV0m6SpJF0gqkFQmaaikUZLe3MHN7yXpM3fftIPb+Z9gZpl8FlFmZpLM3TenOhYAAWDiFIB4ZtZC0p8kXeTuz7r7t+6+0d0nuvuvYmMamdmtZlYSe9xqZo1iyw41syIzu9zMlsWqsGfHlv1R0tWSRscqtOea2R/M7LG4/VdoNZvZWWb2hZmtNbOFZnZq3Otvxq13gJnNip1GMMvMDohb9qqZXWtmb8W2M83MWlfz/rfEf2Vc/Mea2dFm9pmZrTKz38aNH2hm75jZ6tjYO80sK7bs9diwd2Pvd3Tc9n9tZkskPRxfnTOzTrF99Is9zzGzFWZ2aALH7lEzuzz2c27sc/xF7Hnn2Hat0v7+JWlPSRNjMV4Zt8lTzeyr2P5/V8N+HzGzu8zsxdjnO8PMOm3DsfmLmb0lab2kjlviNrMFse1dG/tc3jGzb8zs33GfcSszm2Rmy83s69jPedXEWf5vJnZ818U9NprZI7FlZ5vZx7F9f2Fm58de31XSFEk5cevlVPFv+Bgz+zD2b+JVM+set2yRmV1hZu/FPo/xZrZLbccWQHohSQWq9iNJu0h6roYxv5M0SFIfSb0lDZT0+7jl7SS1kJQr6VxJd5lZK3e/RtJ1ksa7e1N3f6imQGJJwe2Shrl7M0kHSJpfxbjdJL0YG7u7pL9LetHMdo8bdoqksyW1kZQl6Yoadt1O0c8gV9Gk+gFJp0naT9LBkq42s46xsRFJl0pqrehnN1jSLyTJ3X8cG9M79n7Hx21/N0WryufF79jdP5f0a0mPm1kTSQ9LesTdX60h3i1ek3Ro7OdDJH0R+68k/VjSG+7ulfZ3uqSvJI2MxXhj3OKDJHWNvaer45OtKpws6Y+SWkkqlPQXKeFjc7qin0MzSV/GXhuq6Oc9SNKVku6XdKqk9pJ6xvYnRX+XP6zoZ7mnpA2S7qwhzi3v+8bY+20qqbuk5ZL+HVu8TNIISc0V/Tdzi5n1c/dvJQ2TVLJlXXcvid+ume0jaZyksZL2kDRZ0T8AsuKGnRh7fx0k9ZJ0Vm3xAkgvJKlA1XaXtKKWFvSpkv7k7svcfbmiycnpccs3xpZvdPfJktYpmuxsj82SeppZY3cvdfcPqxgzXNICd/+Xu29y93GSPpE0Mm7Mw+7+mbtvUDQZ6VPDPjdK+ou7b5T0pKIJ6G3uvja2/w8VTS7k7nPcfXpsv4sk3acfEsOa3tM17v59LJ4K3P0BSQskzZCUregfBYl4TdLBZpahaFJ6o6QDY8sOiS3fFn909w3u/q6kdxX9g6Q6z7r7zNi/m8f1w+ebyLF5xN0/jC3fGHvtr+7+Tezz/kDSNHf/wt3XKFrN7CtJ7r7S3Z9x9/XuvlbR5Li2z7+cmTWW9Lyix3dybJsvuvvnHvWapGmK/nGSiNGSXnT3l2Lv5WZJjRX9A2uL2929xN1XSZqomv8tAukr1ROmmDgFBGelpNZW88zuHP1Q8VLs55z4bVRKctdLarqtgcQqV6MVPTe2NNZO7pZAPFtiyo17vmQb4lnp7pHYz1uSyKVxyzdsWd/M9om1mJeY2TeKVoqrPJUgznJ3/66WMQ8oWjG8w92/r2WspPIq7DpFk56DJU2SVGJmXbV9Seq2fGbVjU3k2CyuYnuVP+/qPv8mZnafmX0Z+/xfl9TSzBrUEGu8hyR96u5/3fKCmQ0zs+mx0yNWSzpatR/TLSq839j5tYu1/f8WAaQhklSgau9I+k7SsTWMKVG0vbrFnrHXtse3kprEPW8Xv9DdC9z9SEUrip8omrzVFs+WmIq3M6ZtcY+icXVx9+aSfiuptj+9vaaFZtZU0q2KJlB/iLXME/WaoldQyHL34tjzMxRtw291qkQi8eygRI7Njuz/ckWr9PvHPv8tp1jUWv4ws6ti654b91ojSc8oWgFt6+4tFW3Zb9lebbFWeL9mZoqeolAX/xYB/I8gSQWqEGunXq3oeaTHxipVDWPVpS3nK46T9Hsz28OiE5CulvRYddusxXxJPzazPS06aes3WxaYWdvYJJRdJX2vaJUwUsU2Jkvax6KXzco0s9GSeihaSUy2ZpK+kbQuVuW9sNLypZI6brVWzW6TNCd2WaMXJd27ZUFsks6rNaz7mqSLFa0oStKrkn4p6c246nBl2xNjopJ9bJopWlldHUvmr0lkJTMbJmmMpGMrnXKRJamRoueoboqNi79s1VJJu8f+rVbl35KGm9lgM2uoaBL9vaS3t+E9AZBFZ/eH8EgBklSgGu7+d0mXKToZarmi7cqLFT13T5L+LGm2pPckvS9pbuy17dnXS5LGx7Y1RxWTlwxF/0++RNIqRVvWv6hiGysVnehyuaKnK1wpaYS7r9iemLbRFYpOylqraJV3fKXlf5D0aGym94m1bczMRik6qeaC2EuXSepnsasaKFqVe6uGTbymaOK2JUl9U9FK9evVriFdr+gfHavNrKYJZdusDo7NrYqe87lC0nRJUxNcb7SiE5s+jpupf2/svNYxiiabXyt6bF+Iez+fKPpH2hexzyv+NBe5+6eKTrK7IxbTSEUnpZXtwHsEkGas0iRXAAiemc2XNDiW/AHA/6SMFnt6owMvT3UYkqTvpoyd4+7963KfXMwfQL3j7swEB5AeUjSzPgS0+wEAABAcKqkAAAAhMnFbVAAAACAkwVVSLbOxW1azVIeBbdC3+56pDgEAgJ1m7tw5K9x9j1THke7CS1KzmqlR11qvUIOAvDWj1luEAwBQbzRuaJXvEJciRrsfAAAACAlJKgAAAIITXLsfAAAAMVwnFQAAAAgHlVQAAIBQMXEKAAAACAdJKgAAAIJDux8AACBUTJwCAAAAwkGSCgAAgODQ7gcAAAiRcVtUAAAAIChUUgEAAELFxCkAAAAgHCSpAAAACA7tfgAAgEAZ7X4AAAAgHCSpAAAACA7tfgAAgACZaPcDAAAAQSFJBQAAQHBo9wMAAITIYo80RSUVAAAAwaGSCgAAECRj4hQAAAAQEpJUAAAABId2PwAAQKBo9wMAAAABIUkFAABAcGj3AwAABIp2PwAAABAQKqkAAACBopIKAAAABIQkFQAAAMGh3Q8AABAiiz3SFJVUAAAABIckFQAAAMGh3Q8AABAgkzG7HwAAAAgJlVQAAIBAUUkFAAAAAkKSCgAAgODQ7gcAAAgU7X4AAAAgICSpAAAACA7tfgAAgEDR7scOufeaU/Xly9dr9lO/rXbM3648Xh9MuEYzx/9Gfbrllb9+5AHd9e5z/6cPJlyjK84+si7ChaRpBVPVK7+r8rt11k033rDVcnfXZWPHKL9bZw3o20vz5s5NeF0kD8et/uGY1U8cN4SAJHUn+NfE6Rp10V3VLh9yUA912nMP9Rz1R13853G6/bcnSZIyMky3XnWiRl18t/oe92edMHQ/devYrq7CTluRSERjx1ykCROnaN57H+mpJ8fp448+qjCmYOoUfV64QB98vEB33nO/xlx8YcLrIjk4bvUPx6x+4rghFCSpO8Fbcz/XqjXrq10+4pBeemLSTEnSzPcXqUWzxmrXurkG9Nxbny9eoUXFK7VxU0RPFczViEN71VXYaWvWzJnq1KmzOnTsqKysLJ0w+iRNmjihwphJL0zQKaedITPT/oMGac2a1SotLU1oXSQHx63+4ZjVTxy3gFhAjxQgSa0DOW1aqmjJ1+XPi5euVk6blspp00JFS+Nf/1q5e7RIRYhppaSkWHl57cuf5+bmqbi4uNYxJcXFCa2L5OC41T8cs/qJ44ZQJC1JNTM3s7/FPb/CzP6QrP2FrKpznt1dVsWfJl4H8aQ7960/5conplc3JpF1kRwct/qHY1Y/cdzCYmZBPFIhmbP7v5f0UzO73t1XJHE/wSteulp57VqVP89t21Kly9coq2Gm8trGv95KJcvXpCLEtJKbm6eiosXlz4uLi5STk1PrmOycHJWVldW6LpKD41b/cMzqJ44bQpHMdv8mSfdLujSJ+6gXXnztfZ0yYqAkaeC+e+ubdRu0ZMU3mv3hl+q85x7aK2d3NcxsoBOG9NOLr76X4mj/9/UfMECFhQu0aOFClZWV6anxT2r4iGMqjBk+8hg98dg/5e6aMX26mjdvoezs7ITWRXJw3Oofjln9xHFDKJJ9ndS7JL1nZjfWNMjMzpN0niSpYdMkh7TzPXr9WTp4vy5q3bKpCqdeq2vvnayGmQ0kSQ8+/aamvvmhhhyUrw9fuEbrv9uo8//wmCQpEtmsS//6b028+yI1yDA9OmG6Pv5iSSrfSlrIzMzULbfdqZHDhygSiejMs85Rj/x8PXDfvZKkn59/gYYOO1oFUyYrv1tnNWncRPc9+HCN6yL5OG71D8esfuK4hcOUulZ7CKyq80d2yobN1rl7UzP7k6SNkjZIauruf6hpvYwmbbxR1xOTEhOS4+tZd6Y6BAAAdprGDW2Ou/dPdRwNW3fyliOvS3UYkqQVj5xU559JXczuv1XSuZJ2rYN9AQAA4H9A0pNUd18l6d+KJqoAAABIUKpn9adydn9dXSf1b5Ja19G+AAAAUM8lbeKUuzeN+3mppCbJ2hcAAMD/pPSdN8UdpwAAALBjzGyomX1qZoVmdlUVy1uY2UQze9fMPjSzs2vbJkkqAAAAtpuZNVD0sqPDJPWQdLKZ9ag07CJJH7l7b0mHSvqbmWXVtN1kXycVAAAA28PqzW1lB0oqdPcvJMnMnpQ0StJHcWNcUjOLvqGmklYpeuOnalFJBQAAQG1am9nsuMd5cctyJS2Oe14Uey3enZK6SyqR9L6kS9x9c007pJIKAACA2qyo4WL+VZV7K98taoik+ZIOl9RJ0ktm9oa7f1PdDklSAQAAAlVP2v1FktrHPc9TtGIa72xJN3j0VqeFZrZQUjdJM6vbKO1+AAAA7IhZkrqYWYfYZKiTJL1QacxXkgZLkpm1ldRV0hc1bZRKKgAAALabu28ys4slFUhqIOkf7v6hmV0QW36vpGslPWJm7yt6esCv3X1FTdslSQUAAAhUPWn3y90nS5pc6bV7434ukXTUtmyTdj8AAACCQyUVAAAgQCarN5XUZKCSCgAAgOCQpAIAACA4tPsBAABClb7dfiqpAAAACA9JKgAAAIJDux8AACBEVn+uk5oMVFIBAAAQHCqpAAAAgaKSCgAAAASEJBUAAADBod0PAAAQKNr9AAAAQEBIUgEAABAc2v0AAAChSt9uP5VUAAAAhIdKKgAAQKCYOAUAAAAEhCQVAAAAwaHdDwAAECAzo90PAAAAhIQkFQAAAMGh3Q8AABAo2v0AAABAQEhSAQAAEBza/QAAAIGi3Q8AAAAEhEoqAABAqNK3kEolFQAAAOEhSQUAAEBwaPcDAAAEiolTAAAAQEBIUgEAABAc2v0AAAAhMtr9AAAAQFCopAIAAATIJKVxIZVKKgAAAMJDkgoAAIDgBNfu77B3tm54+LepDgPb4Mjb3kx1CNgOUy4+INUhYBtlNqCuAKQXY+IUAAAAEBKSVAAAAAQnuHY/AAAAotK4208lFQAAAOEhSQUAAEBwaPcDAAAEitn9AAAAQECopAIAAITImDgFAAAABIUkFQAAAMGh3Q8AABAgk5SRkb79fiqpAAAACA5JKgAAAIJDux8AACBQzO4HAAAAAkIlFQAAIFDccQoAAAAICEkqAAAAgkO7HwAAIETcFhUAAAAIC0kqAAAAgkO7HwAAIEAmZvcDAAAAQaGSCgAAECSjkgoAAACEhCQVAAAAwaHdDwAAEKg07vZTSQUAAEB4SFIBAAAQHNr9AAAAgWJ2PwAAABAQklQAAAAEh3Y/AABAiIzZ/QAAAEBQqKQCAAAEyMTEKQAAACAoJKkAAAAIDu1+AACAQKVxt59KKgAAAMJDkgoAAIDg0O4HAAAIFLP7AQAAgIBQSQUAAAhUGhdSqaQCAAAgPCSpAAAACA7tfgAAgBAZE6cAAACAoJCk7gTz33pFlxx7sH55zIF6/h93brV81isFuuLEI/Sr0UfqqlOG6ZN5Myss3xyJ6MqTjtINY86oq5DT3sC9W+rxs/tp3Dn76dSBeVWO6ZPXQv84vY/+eWZf3XHivhWWZZj00Ol99Ndje9RFuIh5adpU9d23u3r32Ed/u+mvWy13d/3qskvUu8c+GtS/j+bPmytJKlq8WEcfNVj79c7XgL776u47b6/r0NPWtIKp6pXfVfndOuumG2/Yarm767KxY5TfrbMG9O2leXPnJrwukofjhhDQ7t9BmyMRPXTD7/T7e8Zp97bZ+s2pR6v/IUcpr9M+5WP23f8g9T/0KJmZvvzsI93y6wt063Ovly+f/MSDyu3QRRu+XZuKt5B2Mky6bHAnXfr0B1q+tkwPnNpHbxWu1KJVG8rHNG3UQJcf0UmXP/Ohlq39Xi0bN6ywjRP65ejLleu1axZfoboSiUR0+SW/1IQXC5Sbl6dDDtxfw0eMVLfuP/yhMK1gij4vXKD5H36qWTNn6NIxF+mVN95RZmamrvvrTerTt5/Wrl2rg380QIcPPqLCutj5IpGIxo65SC9OeUm5eXk6aNAAjRhxjLr3+OFzL5gaPWYffLxAM2fM0JiLL9Qbb89IaF0kB8ctHCZm92MHFH4wT+3a7622eXsps2GWDhgySrNeLagwZpcmu5afU/L9hvUVzi9ZubREc998WYN/cnKdxp3OurdrpuLV36l0zffatNn18qfLdVDn3SuMOaLbHnptwQotW/u9JGn1ho3ly/ZomqUfddhNk95fWqdxp7vZs2aqY6dO6tCxo7KysnTcCaM1aeILFca8OPEFnXzq6TIzDdx/kFavXq0lpaVql52tPn37SZKaNWumrt26qaS4OBVvI63MmjlTnTp1Lj9mJ4w+SZMmTqgwZtILE3TKaWfIzLT/oEFas2a1SktLE1oXycFxQyhIUnfQqmVLtHvbnPLnu7fN1qrlS7YaN/O/UzT2Jz/W9WPO1IXX/K389UduukanXfJ7WQaHoq7s0TSrPPmUpOVrv1frplkVxrRv1VjNdsnU7SfuqwdP66MhPdqULxtzWEfd/fpCbfY6CxmSSkuKlZvXvvx5bm6uSksqJpolW43JU0mlMV8uWqT35s9X/4H7JzdgqKSkWHmVjkdx8dbHrPKYkuLihNZFcnDcEIqkZ0Zm9jsz+9DM3jOz+Wb2P/X/DK6tMxXT1rX5gYcP063Pva5f/f0hjb/7JknSnNdfUovdWqtjj15JjxNxEmidNMgwdW3TVFc++6Euf+YDnTmovdq32kUHdGylr9dv1GfLvk1+nKjAvYrvWqU+WG1j1q1bp9NOPkE33Px3NW/efOcHiQp25Jglsi6Sg+MWEpNZGI9USOoJdWb2I0kjJPVz9+/NrLWkrFpWq1d2b5OtlUtLyp+vXFqqVnu0rXZ8j/0G6a6iL/XN16v06fzZmv3aNM17878qK/teG75dq9t/90uN+csddRF62lq+tkxtmjUqf75Hs0Zasa6s4ph1ZVqzYbW+27RZ323arHeL1qjTHruqa5umOrDTbhrUoZWyMjO0a1YD/d+wfXTtlM/q+m2knZzcPBUXLS5/XlxcrHbZORXG5G41pkjZsTEbN27UaScdrxNPOkWjjv1p3QSd5nJz81RU6Xjk5Gx9zCqPyc7JUVlZWa3rIjk4bghFsiup2ZJWuPv3kuTuK9y9pJZ16pVO+X1U+tVCLSv+Sps2luntggnqf+hRFcYs+Wph+V+XX3z8vjZt3KhmLVvplDG/0b0Fc3TX5Bkae8Pd6jngQBLUOvDJkrXKa9lY2c0bKTPDNLjrHnrz81UVxrxZuFK9c5urgUmNMjPUI7uZvly5Qfe9+aWOu3+WTnxwtv4w6VPN/WoNCWod2a//AH1eWKhFCxeqrKxMzzw1XsNHjKww5ugRIzXu8X/J3TVzxnS1aNFC7bKz5e666PyfqWu37vrlJZem6B2kn/4DBqiwcEH5MXtq/JMaPuKYCmOGjzxGTzz2T7m7ZkyfrubNWyg7OzuhdZEcHLewmIXxSIVkT02eJulqM/tM0n8kjXf31yoPMrPzJJ0nSa2zc5Mc0s7VIDNT5/z6z/rLL07R5s2bddio0WrfqaumPfVPSdJRJ5yh6S9P1uuTnlaDzExlNdpFl/71HtofKRRx6Zb/fq6/HddTGRnSix8s1aKV6zWqVztJ0oT3lujLVRs0Y9HXeuTMftrsrknvL9XCletTHHl6y8zM1M233q5jRw7T5khEp595trr3yNdDD9wrSTr35xdoyNCjNW3qFPXusY8aN2mie+5/SJL0zttvadwTjym/5746YGB0AtU1f/qzhgw9OmXvJx1kZmbqltvu1MjhQxSJRHTmWeeoR36+Hrgvesx+fv4FGjrsaBVMmaz8bp3VpHET3ffgwzWui+TjuCEUVtX5Izt1B2YNJB0s6TBJ50u6yt0fqW58px69/YYnpiQ1Juxct7/8RapDwHaYcvEBqQ4B2yizARMsgbrQuKHNcff+qY6jaV437zXm/lSHIUl659eH1PlnkvSLPLp7RNKrkl41s/clnSnpkWTvFwAAoL5L585rUv8sN7OuZtYl7qU+kr5M5j4BAABQ/yW7ktpU0h1m1lLSJkmFip17CgAAAFQnqUmqu8+RxIlvAAAA2yqFM+tDwFn4AAAACE7SJ04BAABg25mYOAUAAAAEhSQVAAAAwaHdDwAAECja/QAAAEBASFIBAAAQHNr9AAAAgUrjbj+VVAAAAISHSioAAECgmDgFAAAABIQkFQAAAMGh3Q8AABAiY+IUAAAAEBSSVAAAAASHdj8AAECATMbsfgAAACAkJKkAAAAIDu1+AACAQKVxt59KKgAAAMJDJRUAACBQGWlcSqWSCgAAgOCQpAIAACA4tPsBAAAClcbdfiqpAAAACA9JKgAAAIJDux8AACBAZuK2qAAAAEBIqKQCAAAEKiN9C6lUUgEAABAeklQAAADsEDMbamafmlmhmV1VzZhDzWy+mX1oZq/Vtk3a/QAAAIGqDxOnzKyBpLskHSmpSNIsM3vB3T+KG9NS0t2Shrr7V2bWprbtUkkFAADAjhgoqdDdv3D3MklPShpVacwpkp51968kyd2X1bZRklQAAADUprWZzY57nBe3LFfS4rjnRbHX4u0jqZWZvWpmc8zsjNp2SLsfAAAgUAF1+1e4e/9qllUVpVd6nilpP0mDJTWW9I6ZTXf3z6rbIUkqAAAAdkSRpPZxz/MklVQxZoW7fyvpWzN7XVJvSdUmqbT7AQAAAmSSLJD/1WKWpC5m1sHMsiSdJOmFSmMmSDrYzDLNrImk/SV9XNNGqaQCAABgu7n7JjO7WFKBpAaS/uHuH5rZBbHl97r7x2Y2VdJ7kjZLetDdP6hpuySpAAAA2CHuPlnS5Eqv3Vvp+U2Sbkp0mySpAAAAgeK2qAAAAEBASFIBAAAQHNr9AAAAITKrF7dFTRYqqQAAAAgOSSoAAACCQ7sfAAAgUGnc7aeSCgAAgPBQSQUAAAiQScpI41IqlVQAAAAEJ7hKavNdGuqorm1THQa2wcieOakOAduh1SG/S3UI2EarXv1zqkPAdkjnSwgBOyK4JBUAAABR6fw3Du1+AAAABIckFQAAAMGh3Q8AABCodD6nmUoqAAAAgkMlFQAAIEBmTJwCAAAAgkKSCgAAgODQ7gcAAAgUt0UFAAAAAkKSCgAAgODQ7gcAAAhU+jb7qaQCAAAgQCSpAAAACA7tfgAAgEBxW1QAAAAgIFRSAQAAAmSSMtK3kEolFQAAAOEhSQUAAEBwaPcDAACEyIyJUwAAAEBISFIBAAAQHNr9AAAAgUrjbj+VVAAAAISHSioAAECgmDgFAAAABIQkFQAAAMGh3Q8AABAgbosKAAAABIYkFQAAAMGh3Q8AABAoZvcDAAAAAam2kmpmd0jy6pa7+5ikRAQAAABJ0clT6aqmdv/sOosCAAAAiKtDqcQAACAASURBVFNtkuruj8Y/N7Nd3f3b5IcEAACAdFfrOalm9iMz+0jSx7Hnvc3s7qRHBgAAkMbMpAyzIB6pkMjEqVslDZG0UpLc/V1JP05mUAAAAEhvCc3ud/fFlV6KJCEWAAAAQFJi10ldbGYHSHIzy5I0RrHWPwAAAJInjS+TmlAl9QJJF0nKlVQsqU/sOQAAAJAUtVZS3X2FpFPrIBYAAABAUmKz+zua2UQzW25my8xsgpl1rIvgAAAA0pmZBfFIhUTa/U9I+rekbEk5kp6SNC6ZQQEAACC9JZKkmrv/y903xR6PqYbbpQIAAGDnMAvjkQrVnpNqZrvFfnzFzK6S9KSiyeloSS/WQWwAAABIUzVNnJqjaFK6JX8+P26ZS7o2WUEBAAAgvVXb7nf3Du7eMfbfyg8mTsX5z7Sp6t+7h/r27Kpbbv7rVsvdXVdePlZ9e3bVAQP7av68uZKk7777TocfPEgH7t9Pg/brpeuu/UMdR56+phVMVa/8rsrv1lk33XjDVsvdXZeNHaP8bp01oG8vzZs7N+F1kTxH7t9F744bqw/GX6YrTtv6xnctm+2i8dedqpmP/lJvPHChenRoU76sRdNd9MSfT9b8J8Zq3uOXaP/89nUZetqaVjBVvfO7qWf3Lrq5mu/a5ZeOUc/uXTSwX2/Nm/fDd+38n5+jvXLbqn+ffesyZIjfkaEwpf52qKHfFlVm1tPMTjSzM7Y8kh1YfRGJRHTFpWP09POTNGPu+3r6qfH65OOPKox5qWCKvihcoLnvf6Lb7rxHl18Svcxso0aN9MKU/+itGXP1xvQ5evmlAs2aOT0VbyOtRCIRjR1zkSZMnKJ5732kp54cp48/qnjMCqZO0eeFC/TBxwt05z33a8zFFya8LpIjI8N06+UjNeryR9X31Nt0whG91G3vPSqMufKMQ/XuglINPPMOnXvtU7p57IjyZTePHa5pMxaozym3auCZd+qTL5fX9VtIO5FIRJdecrGenzhZc9/9UE+Nf7LK71phYaHe/+gz3XnPfbrk4l+ULzv9jLP0/KQpdR122uN3JEKRyCWorpF0R+xxmKQbJR2T5LjqjTmzZ6pjp07au0NHZWVl6bjjT9TkSS9UGDN50kSddOrpMjMNGDhIa9as0ZLSUpmZmjZtKknauHGjNm7cJFOKzk5OI7NmzlSnTp3VoWP0mJ0w+iRNmjihwphJL0zQKaedITPT/oMGac2a1SotLU1oXSTHgO55+rxolRaVfK2NmyJ66uX3NOLg7hXGdNu7jV6d87kk6bOvVmiv7JZq02pXNWvSSAf13luPTJwtSdq4KaI1676r8/eQbmbPqvh9Of7E0Vt/1yZO0Kmx348D9x+kNauj3zVJOujgH2u3VrtVtWkkEb8jEYpEKqnHSxosaYm7ny2pt6RGSY2qHiktKVFu7g9tw5zcPJWWlFQaU6zcvLy4MbkqLSmWFP2r86D991OXvbJ12ODB6j9w/7oJPI2VlBQrL++HY5abm6fi4uJax5QUFye0LpIjZ4/mKlq2pvx58bJvlLtHiwpj3i8s1ahDekiS+nfP055tWyq3TQt1yN1NK1av1/2/O07vPHyR7r7qJ2qyS8M6jT8dlRRX/N2Xm5unkpLK37US5bWP+07lbT0GdYvfkQEJYFZ/Kmf3J5KkbnD3zZI2mVlzScsk1XpOqpm9amZDKr021szu3r5Qw+RexdW4Kh3NqsZsuTBugwYN9OaMOfpwwZeaM3uWPvrwg6TEiR/UdDxqG5PIukiOqj7nysfj5n+9rpbNGmv6IxfrwuMH6d0FpdoU2azMBhnqs0+2Hnhuhn509l1av6FMV5x+SF2FnrZ25LuG1OF3JEJR621RJc02s5aSHlB0xv86STMTWG+cpJMkFcS9dpKkX21rkCHLyc1VcfHi8uclxUXKzs6uNCZPxUVFcWOK1S47p8KYli1b6qCDD9HLLxWoR37P5Aad5nJz81RU9MMxKy4uUk5OTq1jsnNyVFZWVuu6SI7iZWuU1+aHymlum+YqWfFNhTFr13+v8697tvz5J09foUUlX6vJLg1VvPwbzfoo+j187tUPdPlpJKnJlptX8XdfcXGRsrMrf9dyVbQ47jtVtPUY1C1+R4YlnZP8Wiup7v4Ld1/t7vdKOlLSmbG2f22eljTCzBpJkpntregdq97c/nDD02+/Afq8sFCLFi1UWVmZnnn63xo2fGSFMcOGj9CTj/9L7q5ZM6erefPmapedrRXLl2v16tWSpA0bNui1V15Wl326puJtpJX+AwaosHCBFi2MHrOnxj+p4SMqnmY9fOQxeuKxf8rdNWP6dDVv3kLZ2dkJrYvkmP1JsTrn7a69slupYWYDnTC4l15885MKY1o03UUNMxtIks4e2V9vzl+kteu/19JV61S0bI267NlaknTofp30yaJldf4e0s1+/St+X57+9/itv2sjjtHjsd+PM2dMV/MWLbb6Qx91i9+RCEVNF/PvV9Myd59b3XJJcveVZjZT0lBJExStoo73Kvvj9VdmZqZu+vttOu6YoxWJRHTaGWepe498/eOB+yRJ5/z8fB019Gi9VDBVfXt2VZMmTXTXvQ9KkpYsKdWFPz9Hkc0R+ebNOvanx2vo0SNq2h12gszMTN1y250aOXyIIpGIzjzrHPXIz9cD990rSfr5+Rdo6LCjVTBlsvK7dVaTxk1034MP17guki8S2axLb5moiX8/Sw0amB6dNFcfL1ymnx07UJL04PMz1W2vPfTg/x2vyGbXJ4uW6YLrf6iqXnbLJD18zYnKymygRSWrdN51z6TqraSNzMxM/f3WO3TM8KGKbI7ojDPPjn7X7o99186LfdemTlbP7l3UpHET3fvgP8rXP/O0U/T6669q5YoV6tyhvX5/9R901tnnpurtpA1+RyIUVl3OaGav1LCeu/vhtW7c7DRJw939ZDObL+mcqpJbMztP0nmS1L79nvu9/+kXCQWPMDRq2CDVIWA7tDrkd6kOAdto1at/TnUI2A7p3K6trxo3tDnu3j/VcbTp3NNH3/RUqsOQJN350x51/plUW0l198N2wvafl/T3WFW2cXXVV3e/X9L9ktS3X///qUorAAAAtl1CF/PfXu6+TtKrkv6h6EQqAAAAoFaJzO7fUeMkPavoOakAAABIgCm9TxdJepLq7s9J3EYJAAAAiUvktqhmZqeZ2dWx53ua2cDkhwYAAIB0lUgl9W5JmyUdLulPktZKekbSgCTGBQAAkPYy0rgXnUiSur+79zOzeZLk7l+bWVaS4wIAAEAaSyRJ3WhmDSS5JJnZHopWVgEAAJBE6VxJTeQSVLdLek5SGzP7i6K3Nb0uqVEBAAAgrdVaSXX3x81sjqTBis7SP9bdP056ZAAAAEhbtSapZranpPWSJsa/5u5fJTMwAACAdGbGdVJr86Ki56OapF0kdZD0qaT8JMYFAACANJZIu3/f+Odm1k/S+UmLCAAAAGlvm+845e5zzYxrpAIAACRZOs/uT+Sc1MvinmZI6idpedIiAgAAQNpLpJLaLO7nTYqeo/pMcsIBAADAFmk8b6rmJDV2Ef+m7v6rOooHAAAAqP5i/maW6e4RRdv7AAAAQJ2pqZI6U9EEdb6ZvSDpKUnfblno7s8mOTYAAIC0ZZIy0rjfn8g5qbtJWinpcP1wvVSXRJIKAACApKgpSW0Tm9n/gX5ITrfwpEYFAACAtFZTktpAUlNVTE63IEkFAABIsmonD6WBmpLUUnf/U51FAgAAAMTUlKSm75m6AAAAAUjjeVM1VpEH11kUAAAAQJxqk1R3X1WXgQAAAABbJHIJKgAAANQxM0vr66Sm86QxAAAABIokFQAAAMGh3Q8AABCoNO72U0kFAABAeEhSAQAAEBza/QAAAIHKoN0PAAAAhINKKgAAQIBM4jqpAAAAQEhIUgEAABAc2v0AAACBSuNuP5VUAAAAhIckFQAAAMGh3Q8AABAi4zqpAAAAQFCopAIAAATKlL6lVCqpAAAACA5JKgAAAIJDux8AACBA0duipjqK1KGSCgAAgOCQpAIAACA4tPsBAAACRbsfAAAACAiVVAAAgECZpW8plUoqAAAAgkOSCgAAgOAE1+7fGNmspWu+T3UY2AZ7tm6S6hCwHRYX/DHVIWAbdbz42VSHgO2w8K7jUh0C6imukwoAAAAEhiQVAAAAwQmu3Q8AAABJJqXx5H4qqQAAAAgPSSoAAACCQ7sfAAAgUBlp3O+nkgoAAIDgUEkFAAAIENdJBQAAAAJDkgoAAIDgkKQCAAAEyiyMR+1x2lAz+9TMCs3sqhrGDTCziJkdX9s2SVIBAACw3cysgaS7JA2T1EPSyWbWo5pxf5VUkMh2SVIBAACwIwZKKnT3L9y9TNKTkkZVMe6Xkp6RtCyRjTK7HwAAIEimDAUzvb+1mc2Oe36/u98f+zlX0uK4ZUWS9o9f2cxyJf1E0uGSBiSyQ5JUAAAA1GaFu/evZllVmbRXen6rpF+7e8QSvEEBSSoAAECATIlNWgpAkaT2cc/zJJVUGtNf0pOxBLW1pKPNbJO7P1/dRklSAQAAsCNmSepiZh0kFUs6SdIp8QPcvcOWn83sEUmTakpQJZJUAAAA7AB332RmFys6a7+BpH+4+4dmdkFs+b3bs12SVAAAgBBZ/bktqrtPljS50mtVJqfuflYi2+QSVAAAAAgOSSoAAACCQ7sfAAAgUBn1ZHp/MlBJBQAAQHBIUgEAABAc2v0AAAABqkcX808KKqkAAAAIDpVUAACAQDFxCgAAAAgISSoAAACCQ7sfAAAgUGnc7aeSCgAAgPCQpAIAACA4tPsBAAACZErvamI6v3cAAAAEikoqAABAiEyyNJ45RSUVAAAAwSFJBQAAQHBo9wMAAAQqfZv9VFIBAAAQIJJUAAAABId2PwAAQIBMUgaz+wEAAIBwUEkFAAAIVPrWUamkAgAAIEAkqQAAAAgO7X4AAIBApfG8KSqpAAAACA9J6k7w+n+nachBfXTkj/bV/XfcvNXyzxd8qtEjDlPPvVrpoXturbDs8AHdNfKwARp1xCD9dMhBdRVy2ptWMFW98rsqv1tn3XTjDVstd3ddNnaM8rt11oC+vTRv7tyE10XyvPxSgQb1zdeA3t10299u3Gq5u+s3vxqrAb276ZBBffXu/B+O25rVq3X2aaP1o349dcB++2rWjHfqMvS0dVh+W73xx6P09rVDdPGQfbZafuFR++il3w/WS78frFeuPkJF9/xULZs0lCT97PDOeuXqI/TqNUfq54M713XoaY3fkQgB7f4dFIlE9KffXqaHx09U2+xcHT/sYB1+1HB17tq9fEzLVq30uz/frJenTKxyG48+PUW77d66rkJOe5FIRGPHXKQXp7yk3Lw8HTRogEaMOEbde/QoH1MwdYo+L1ygDz5eoJkzZmjMxRfqjbdnJLQukiMSieiqy8foqQlTlJObp6MOGaShw0eoa7cfPvv/TJuqLz4v1Mz5H2vOrBm68tKLVfDK25Kk3155qQ4/4ig9/Nh4lZWVacP69al6K2kjw6TrTu6j0be+qdKv12vKbw7XtPdK9Vnp2vIx90z7TPdM+0ySdGSvbJ03uLNWr9+orjnNdepBe+vo619RWWSznhhzkP7z/hItXLYuVW8nbfA7MiQmS+N+P5XUHfTevNnaa++Oar9XB2VlZWn4qOP1csGkCmN2b91Gvfrsp8yGDVMUJeLNmjlTnTp1VoeOHZWVlaUTRp+kSRMnVBgz6YUJOuW0M2Rm2n/QIK1Zs1qlpaUJrYvkmDt7pvbu2El7d4h+9sceN1pTJlX8w2/qiy9o9MmnyczUf+AgrVm9RkuWlGrtN99o+ttv6rQzz5EkZWVlqUXLlql4G2mlb4fdtGjZt/pqxbfaGHFNmF2kIb1zqh1/7IA8PT9rsSSpS7tmmrNwlTZsjCiy2TX9s+Ua1qf6dbHz8DsSoSBJ3UFLl5SoXW5e+fO22blauqQ08Q2Y6dyTjtFPjzpQ4//1jyREiMpKSoqVl9e+/Hlubp6Ki4trHVNSXJzQukiO0tIS5cZ913Jyc1VaWvGzLy0pUU6lMUtKirVo0RfavXVr/fKCc3XYgf019qLz9O2339ZZ7OmqXcvGKv76h4p16dcb1K5l4yrHNm7YQIflt9OLc6PH9NOSbzSoS2u12jVLjRs20OH7tlPOblWvi52L35EIRdKTVDOLmNn8uMfeyd5nXXL3rV7bltL8uBde1nMvva0HnnhOjz9yn2a98+bODA9VSOSYVTdmR483tt+OHLfIpk16b/48nf2z8/XKW7PVZNdddfvftz6nFTtXVd8M19bHSJKO7J2tWZ+v1Or1GyVJC5as1V0Fn2n82IP0xCUH6qPFaxSJVL0udi5+R4bDFE3UQnikQl2ck7rB3fvUwX5Sol12rpYUF5U/X1parDZt2yW8ftt22ZKipwQcOewYvTd/tgb8iAlUyZSbm6eiosXlz4uLi5STk1PrmOycHJWVldW6LpIjJydXxXHftZLiYrVrV/Gzz8nNVUmlMW2zc2RmysnN034D9pckjRx1HElqHShdvUG5rZqUP89u1VhLV39X5dhj++fp+ZmLK7w27q1FGvfWIknSb47NV8nXG5IWK37A70iEgnb/Dtq3z35atPBzLf5qkcrKyvTihKd1+JDhCa27fv23WrdubfnPb732srp05eTyZOs/YIAKCxdo0cKFKisr01Pjn9TwEcdUGDN85DF64rF/yt01Y/p0NW/eQtnZ2Qmti+Tou98ALfy8UF8uin72zz8zXkOHj6gwZsjRIzV+3GNyd82eOV3NWzRXu3bZatu2nXJy81T42aeSpDde+6+6dute1W6wE81f9LU6tGmq9rs3UcMGplH981TwbslW45rtkqlB++yhqZWW7d6skSQpt1VjHd03t/x8VSQXvyPDYmZBPFKhLiqpjc1sfuznhe7+k8oDzOw8SedJUk5u+8qLg5aZmamrr/ubfnbyKEUiER130hnq0rWHxj36oCTp5DN/puXLlui4oQdr3dq1ysjI0KMP3KXJr83R16tW6qJzTpIkRTZFNOInJ+rHhx+VyreTFjIzM3XLbXdq5PAhikQiOvOsc9QjP18P3HevJOnn51+gocOOVsGUycrv1llNGjfRfQ8+XOO6SL7MzExdf/NtOvHY4dq8OaKTTz9L3brn65GH7pMknXXu+TpyyDD9Z9oUDezdTY0bN9bt9zxYvv71N9+qC352hjaWlWmvvTtWWIbkiGx2/fbJ+Rp3yUFqkGF68q1F+qx0rc74cQdJ0j9fXyhJGtY3V699tFQbyiIV1n/o/EFqtWuWNkY26zfj5mlN7FQAJBe/IxEKq+r8kZ26A7N17t400fE9e/fzZws4L7M+2bN1k9oHITjrvtuU6hCwjfa9nFnS9dHCu45LdQjYRo0b2hx375/qODr16O3XPzEl1WFIkkb3za3zz4TrpAIAAAQqnaedcU4qAAAAgkOSCgAAgOAkvd2/LeejAgAAIMbS+zqzVFIBAAAQHCZOAQAABGjLHafSVTq/dwAAAASKJBUAAADBod0PAAAQKCZOAQAAAAEhSQUAAEBwaPcDAAAEKn2b/VRSAQAAECCSVAAAAASHdj8AAECg0nhyP5VUAAAAhIdKKgAAQICit0VN31IqlVQAAAAEhyQVAAAAwaHdDwAAECgmTgEAAAABIUkFAABAcGj3AwAABMlkzO4HAAAAwkElFQAAIFBMnAIAAAACQpIKAACA4NDuBwAACBC3RQUAAAACQ5IKAACA4NDuBwAACJExux8AAAAICpVUAACAQFFJBQAAAAJCkgoAAIDg0O4HAAAIlHGdVAAAACAcJKkAAAAIDu1+AACAAJmkjPTt9lNJBQAAQHhIUgEAABAc2v0AAACBYnY/AAAAEBAqqQAAAIHitqgAAABAQEhSAQAAEBza/QAAAIFi4hQAAAAQEJJUAAAABId2PwAAQIC4LSoAAAAQGCqpAAAAQbK0njgVXJKalZmh9rs3TnUYwP+8prsE9/VHLRbedVyqQ8B2aDXwl6kOAaiXaPcDAAAgOJRSAAAAQmTcFhUAAAAICkkqAAAAgkO7HwAAIFBp3O2nkgoAAIDwUEkFAAAIUPSOU+lbS6WSCgAAgOCQpAIAACA4tPsBAAAClb7NfiqpAAAACBBJKgAAAIJDux8AACBUadzvp5IKAACA4JCkAgAAIDi0+wEAAAJladzvp5IKAACA4FBJBQAACFQa3xWVSioAAADCQ5IKAACA4NDuBwAACFQad/uppAIAACA8JKkAAAAIDu1+AACAUKVxv59KKgAAAIJDJRUAACBAJu44BQAAAASFJBUAAADBod0PAAAQIuO2qAAAAEBQSFIBAAAQHNr9AAAAgUrjbj+VVAAAAISHJBUAAADBod0PAAAQqjTu91NJBQAAQHCopAIAAATJuC0qAAAAEBKSVAAAAASHdj8AAECguC0qAAAAsJ3MbKiZfWpmhWZ2VRXLTzWz92KPt82sd23bJEkFAADAdjOzBpLukjRMUg9JJ5tZj0rDFko6xN17SbpW0v21bZd2PwAAQIBM9eYyqQMlFbr7F5JkZk9KGiXpoy0D3P3tuPHTJeXVtlEqqQAAAKhNazObHfc4L25ZrqTFcc+LYq9V51xJU2rbIZVUAACAUIVTSl3h7v2rWVZVlF7lQLPDFE1SD6pthySpAAAA2BFFktrHPc+TVFJ5kJn1kvSgpGHuvrK2jdLuBwAAwI6YJamLmXUwsyxJJ0l6IX6Ame0p6VlJp7v7Z4lslCR1J5hWMFW987upZ/cuuvnGG7Za7u66/NIx6tm9iwb266158+YmvC6SY1rBVP1/e/ceb2VVJnD89wiY4B1FhXPQDCwS84a37GZOFl7JUUfLyTTNS5Jmdpumj2POpztW01heIk3LFB2dQEUoLdMuCIJXMAXTSQ5mpmFmKnp45o/9IuccuRzl7LPX5vy+fPaHc9693v2uvRfs/eznWe+7dhz9JkaPGsk3VjJmn/zEaYweNZLdd9mRO+fM6fa+qh/Hrfk4Zs1pv73fzN3XfoH7Jp/Fp47d7xX3b7LhQCZNOIGZkz7HbZd9iu1HDH35vlM/8C7uuOrfmH315xn/wX16sddrpyjkz6pk5kvAeGA6cD9wVWbOjYiTI+LkqtlZwGbA9yLiroi4Y3XP3SB1DbW3t3PG6eP56XVTmXP3XK6edCX3z5vXqc30aTeyYMEC7p33IOedfyGnj/9Yt/dVz2tvb+cTp53K5Otu5M575nH1lVescMweWjCf++6fz3nnX8Rp40/p9r6qD8et+ThmzWmddYJvf/YIxn38fHY57EscMXYMo7bdqlObzxz/Xu5+sI09jvwqx5/1IyZ8+jAAth8xlOMO3Zt3HDOBPY76Kvu/YwdGDB/SiKehXpaZUzPzjZk5IjO/VG27IDMvqH4+ITM3zcydq9vK5re+zCB1Dd0xayYjRoxk2ze8gXXXXZfD/+VIrr9ucqc21183maOP/hARwR577sXTixfz2GOPdWtf9bxZMzu/7kccedQrx2zKZD74r8cQEey51148/XRtzLqzr+rDcWs+jllz2n2HbXho4V94pO1JXnypnaunz+agfd7Sqc2obYdyy8wHAHjwkcfZZuhgthi8IaO23ZKZ9z7Cc8+/SHv7Um6bPZ9x++7YiKehtYBB6hpa1NZGS+vyS321tLSyaFFb5zaLFtE6fPl84pbWWpvu7Kuet2hRG62tHcajpZW2tq5j9so2i9raurWv6sNxaz6OWXMaNmQTFv7pry//3vbnxbRssUmnNvfOb2PcvrUFg3YbvQ1bDx1My5abMPehx3j7riMZvPEgBq43gLFvH03rlpv2av/XNhFl3Bqhrmf3R0QrtRUItgf6AVOBMzPzhXoetzdlvvIKC9FlNFfWpjv7quc5Zs3JcWs+jllzWtHL3HU8JlzycyZ8+jBmXPFZ5i5YxN0PLOSll5bywMOPc+4Pf8713xvPs8+9wD0PtvFS+9Je6rnWNnULUqP2bnItcH5mjquWzLoI+Dpwer2O29taWltpW7jw5d/b2hYydOiwzm1aWlj46PJr3LYtrLV5ccmS1e6rntfS0srChR3Go20hw4Z1HbNXthk6bBhLlixZ7b6qD8et+Thmzantz4tp3Wp59rNli01Y9MTTndo88+zznHT25S///vvrz+aRRbUrCl06eQaXTp4BwBfHH0zb44t7oddrr7781aye5f59gecz8xKAzGwHzgCOiYgN6njcXjVmt91ZsGA+jzz8MEuWLOF/rprEgQcd0qnNgQcdwuWX/4jMZObtM9ho440ZOnRot/ZVz9tt986v+9WTrnzlmB18CD/58WVkJrfPmMFGG9XGrDv7qj4ct+bjmDWnO+b+kZHDh7DNsM0Y0L8fR7xvDDf86t5ObTbeYCAD+vcD4LhD9+bXcx7imWefB2DIprWP+OFbbcq4d+/EVdNWexK3tEL1LPePBmZ33JCZf4uIR4CRwF3LtldLa50IMHzrrevYpZ7Xv39/vvnt/+aQA8fSvrSdYz58HNuPHs33L7oAgI+eeDJj9z+A6dOmssObt2PQwEFcMPHiVe6r+urfvz/f+q/zOPjA99He3s6Hj/1IbcwurMbspGrMbpzK6FEjGTRwEBdOvGSV+6r+HLfm45g1p/b2pZzxtau57rsfo986waVTZnD/H/7ECYe9DYCJ1/yGUW/YkonnfIj29uT3D/+Jk7+4PKt6xYQTGLzxIF58aSmf+NpVLH7muUY9FTW5WNG8nx554IjTgW0y85Ndtt8FHJuZd61ov13H7Ja/mTGrLn1SfThPTJJWbtM9Pt7oLuhVev7O82Z35xJJ9TZ6p11z0tRbG90NAN7SumGvvyb1LPfPBTo9mYjYCNgSeKCOx5UkSVKTq2eQejMwKCKOAahOnDoXOC8zzf1LkiRppeoWpGZtHsGhwOERMR94Eli6xMChSAAADAZJREFUbBUCSZIkrVqjl0PtzrKo9VLXi/ln5qOZeUhmbgccAIyNiDH1PKYkSZKaX10v5t9RZv4W2Ka3jidJkqTm1WtBqiRJkrovaNySpCWoa7lfkiRJei3MpEqSJBWqDydSzaRKkiSpPAapkiRJKo7lfkmSpFL14Xq/mVRJkiQVxyBVkiRJxbHcL0mSVKhGLUlaAjOpkiRJKo6ZVEmSpEK54pQkSZJUEINUSZIkFcdyvyRJUqH6cLXfTKokSZLKY5AqSZKk4ljulyRJKlUfrvebSZUkSVJxDFIlSZJUHMv9kiRJBQpcFlWSJEkqiplUSZKkEoXLokqSJElFMUiVJElScSz3S5IkFaoPV/vNpEqSJKk8BqmSJEkqjuV+SZKkUvXher+ZVEmSJBXHTKokSVKRwhWnJEmSpJIYpEqSJKk4lvslSZIK5bKokiRJUkEMUiVJklQcy/2SJEkFCvr0ZVLNpEqSJKk8ZlIlSZJK1YdTqWZSJUmSVByDVEmSJBXHcr8kSVKhXBZVkiRJKohBqiRJkopjuV+SJKlQLosqSZIkFcQgVZIkScWx3C9JklSoPlztN5MqSZKk8hSXSb1zzuy/DFp3nf9rdD/qZHPgL43uhF4Vx6z5OGbNyXFrPmvzmG3T6A4AEH37xKnigtTMHNLoPtRLRNyRmbs1uh/qPses+Thmzclxaz6OmerNcr8kSZKKU1wmVZIkScv03Xq/mdTedVGjO6BXzTFrPo5Zc3Lcmo9jprqKzGx0HyRJktTFjruMyam/+F2juwHA8MGvm93bc5At90uSJBUo6Ntn91vul7RWiIjNG90HSVLPMUiVViAiBjW6D+q+iNgG+HpEtDa6L1JfENGX83u9Kwq5NYJBap1FxNYRsX6j+6Hui4gDgC9HxPBG90XdtgHQAmwBEBG+tzWBiNiq0X3Qa+b/MdWd/8jqKCK2BM4ETjFQbQ4RcRDwFeCWzHy00f1R92TmXOAW4IKI2Cgzlza4S1qNiDgQmBIRa+0CLmujiPhqRPwAuDgiTm90f7R2M0itryeAWcAw4CMGqmWrsjpnAidk5k8jYt2IGBQRrRGxXqP7p84iYnBEbNBh03eAOcCu1f2+vxUqIsYCnwPOyswnImJAo/uk1YuIS4DtgSuAycD4iPhKRGzU2J6t3SLKuDWCb+J1EBHbRcSbqmzO5cAvgTcCx3f5UFVZXgBeBJ6vgtLPA1OojeH5ETG4kZ3TchGxCXANcHZEjAPIzGeBp4ATqt/Nphao+n80FTg3M6dFxAhgYvWlw3mOhYqI/YCWzDwkM2/KzGuBfwL2BD7b2N5pbWWQ2sMiYjPgAeC2iDgVOAm4Abgd2Ag4wZNyirUYmA5MABYArweuBD5Dbd742xvWM3WSmYupBaP3UPsC8eXqQ/QsYKuI+EBDO6iVysyngIOBsyJiR2oXhL8zM59KL9xduoUAETEgIvpn5h+BY4D3V2Mp9Sivk9rDMvPJiHgPcBO1LwE7AZOAvwNLgE2AFyNiYma+0LieqqvMzIi4EPgtMByYvGyMIuJEal8yVIjMfAh4KCJmAIcDpwKfpPYFYwy1kqQKlJk3REQ7cBfw+cz8djU9Iw1Ui/UosGtE7JWZMwAiYv3MXBgRc6h9xqkOwmVR1ZMy8xfA+4CPAeOpfXDeAmwNvAc4BXCOY4Ey8++Z+bvMvKpDgHoEtS8bZSz7oU4y80Hga5n5fmA2sCNwbERs2NieaVUycxq198ljI2LjanpGvwZ3Syv3ALUvfkdGxM7w8hQbgM2Br1RXRpF6jJnUOsnMn0fEp4D7gL0y89KImAIMAAZl5tON7aFWJyKGAkcCHwWOrDJ3KtNSgMz8QnVVDTLzmcZ2SatTvU+eAcyMiLdWUwFUoKrSdBm1isW/R8SN1E4MPgcYSG3qje+R9dB3E6kGqfVUlbSWAjOqN+AnG90nvSqLgfnAuMxc0OjOaOWqD9DImscb3R91X2beGBHrAjdFxG5Y8i9WZj4WEd8E3gucBuwO/CEzz2xsz7S2Mkitsy5vwGM847h5ZOZz1E56UxMwsGlemTk5Im72/bF8Vbb7yoi4NjOXLNseEes4fuppBqm9wDdgSVq1zPTEm+by4rIfqiqGn2910oer/Z441Vt8A5YkrS06Vi6sYqheDFIlSZJUHMv9kiRJBWrkkqQlMJMqSZKk4hikSlojEdEeEXdFxH0RcfWaLPsbET+MiMOrnydGxParaLtPROz9Go7xSERs3t3tXdq8qrnlEXF2db1kSdKrZJAqaU09l5k7Z+YO1Jb+PbnjnRHxmlYRyswTMnPeKprsA7zqIFWSmkkU8qcRDFIl9aTbgJFVlvOXEfET4N6I6BcR34iIWRFxT0ScBLVL10TEeRExLyJuALZY9kARcUt1cXciYmxEzImIuyPi5oh4PbVg+Iwqi/uOiBgSEddUx5gVEW+r9t0sIn4WEXdGxIV044ouEfHTiJgdEXMj4sQu951b9eXmiBhSbRsREdOqfW6LiFE98WJKUl/miVOSekRE9Af2B6ZVm/YAdsjMh6tA7+nM3D0iXgf8JiJ+BuwCvAl4C7AlMA+4uMvjDgG+D7yzeqzBmflURFwA/D0zJ1TtfgJ8KzN/HRFbA9OBNwP/Afw6M8+JiAOBTkHnSnykOsZAYFZEXFOtGLc+MCczz4yIs6rHHg9cBJycmfMjYk/ge8C+r+FllKTO+vCJUwapktbUwIi4q/r5NuAH1MrwMzPz4Wr7e4Edl803BTYGtgPeCVyRme3Aooj4xQoefy/g1mWPtYr13d8DbB/LT4XdKCI2rI7xz9W+N0TEX7vxnE6LiEOrn4dXfX0SWApMqrb/GLg2Ijaonu/VHY79um4cQ5K0CgapktbUc5m5c8cNVbD2bMdNwMczc3qXdgcAq7sQeHSjDdSmL721Ws62a1+6fbHxiNiHWsD71sz8R0TcAqy3kuZZHXdx19dAkrRmnJMqqTdMB06JiAEAEfHGiFgfuBU4qpqzOhR49wr2/R3wrojYttp3cLX9GWDDDu1+Rq30TtVuWdB4K3B0tW1/YNPV9HVj4K9VgDqKWiZ3mXWAZdngD1KbRvA34OGIOKI6RkTETqs5hiR1SxRyawSDVEm9YSK1+aZzIuI+4EJqlZz/BeYD9wLnA7/qumNmPkFtHum1EXE3y8vt1wGHLjtxCjgN2K06MWsey68y8EXgnRExh9q0gz+upq/TgP4RcQ/wn8CMDvc9C4yOiNnU5pyeU20/Gji+6t9cYFw3XhNJ0iqES+5KkiSVZ+ddx+RNt97e6G4AMGTDAbMzc7fePKZzUiVJkgrlsqiSJElSQcykSpIkFalxqz2VwEyqJEmSimOQKkmSpOJY7pckSSpQ4IlTkiRJUlEMUiVJklQcg1RJkiQVxyBVkiRJxTFIlSRJUnE8u1+SJKlQnt0vSZIkFcRMqiRJUqFcFlWSJEkqiEGqJEmSimO5X5IkqUThiVOSJElSUQxSJUmSVBzL/ZIkSQWK6tZXmUmVJElSccykSpIklaoPp1LNpEqSJKk4BqmSJEkqjuV+SZKkQrksqiRJklQQg1RJkiQVx3K/JElSoVwWVZIkSSqImVRJkqRC9eFEqplUSZIklccgVZIkScWx3C9JklSqPlzvN5MqSZKk4hikSpIkqTiW+yVJkgrlsqiSJElSQQxSJUmStEYiYmxEPBARCyLicyu4PyLiO9X990TErqt7TMv9kiRJBQqaY1nUiOgHfBfYD1gIzIqIKZk5r0Oz/YHtqtuewPnV3ytlJlWSJElrYg9gQWb+ITOXAFcC47q0GQdcljUzgE0iYuiqHtRMqiRJUoHmzJk9feCA2LzR/aisFxF3dPj9osy8qPq5BXi0w30LeWWWdEVtWoDHVnZAg1RJkqQCZebYRvehm1Y0KSFfQ5tOLPdLkiRpTSwEhnf4vRVY9BradGKQKkmSpDUxC9guIraNiHWBo4ApXdpMAY6pzvLfC3g6M1da6gfL/ZIkSVoDmflSRIwHpgP9gIszc25EnFzdfwEwFTgAWAD8AzhudY8bmaucDiBJkiT1Osv9kiRJKo5BqiRJkopjkCpJkqTiGKRKkiSpOAapkiRJKo5BqiRJkopjkCpJkqTi/D8hl41uCkEVlAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import itertools\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "# Compute confusion matrix\n",
    "cnf_matrix = confusion_matrix(y_valid.argmax(axis=1), y_pred.argmax(axis=1))\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure(figsize=(10, 10))\n",
    "plot_confusion_matrix(cnf_matrix, classes=['N', 'S', 'V', 'F', 'Q'],normalize=True,\n",
    "                      title='Confusion matrix, with normalization')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
